#! /usr/bin/env python
# -*- coding: utf-8 -*-


# Parlons Culture - Tools to generate and handle "Parlons Culture" OAI-PMH files
# By: Emmanuel Raviart <eraviart@easter-eggs.com>
#
# Copyright (C) 2010 Easter-eggs
# http://code.google.com/p/parlons-culture/
#
# This file is part of "Parlons Culture" project.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the <organization> nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""Extract themes and sub-themes form OAI-PMH XML files."""


import codecs
import logging
from optparse import OptionParser
import os
import sys
import urllib
import urllib2

from lxml import etree


app_name = os.path.splitext(os.path.basename(__file__))[0]
log = logging.getLogger(app_name)
namespaces = dict(
    dc = 'http://purl.org/dc/elements/1.1/',
    oai = 'http://www.openarchives.org/OAI/2.0/',
    oai_dc = 'http://www.openarchives.org/OAI/2.0/oai_dc/',
    )


def main(argv = None):
    if argv is None:
        argv = sys.argv
    parser = OptionParser("usage: %prog [options] directory")
    parser.add_option('-v', '--verbose', action = 'store_true', default = False, help = "Display more messages")
    options, arguments = parser.parse_args(argv[1:])
    if len(arguments) != 1:
        parser.error("Incorrect number of arguments")
    logging.basicConfig(level = logging.DEBUG if options.verbose else logging.WARNING, stream = sys.stdout)
    dir = arguments[0]
    if not os.path.exists(dir):
        os.makedirs(dir)

    keywords = {}
    locations = {}
    themes = {}
    for organization_name, repo_url, repo_set in [
            (u'BPI', 'http://archives-sonores.bpi.fr/oai.php', None),
            (u'Cinémathèque française', 'http://www.cineressources.net/oai/serveurOAI.php', 'Conferences'),
            (u'Cité Chaillot', 'http://portaildocumentaire.citechaillot.fr:8080/oaicat/OAIHandler', 'VOD'),
            (u'Jeu de Paume', 'http://www.jeudepaume.org/parlons_culture_xml.php', None),
            (
                u'Les Arts décoratifs',
                'http://parlons-culture.easter-eggs.com/depots-statiques/les-arts-decoratifs-parlons-savoirs.xml',
                None,
                ),
            (u'RMN', 'http://parlons-culture.easter-eggs.com/depots-statiques/rmn-enregistrements.xml', None),
            ]:
        page_number = 0
        resumption_token = None
        while True:
            page_number += 1
            if resumption_token is None:
                params = dict(verb = 'ListRecords', metadataPrefix = 'oai_dc')
                if repo_set is not None:
                    params['set'] = repo_set
            else:
                params = dict(verb = 'ListRecords', resumptionToken = resumption_token)
            log.info('Loading {0} with {1}'.format(repo_url, params))
#            response = urllib2.urlopen(repo_url, urllib.urlencode(params))
            response = urllib2.urlopen('{0}?{1}'.format(repo_url, urllib.urlencode(params)))
            oai_pmh_doc = etree.parse(response)
            response.close()
            for oai_record in oai_pmh_doc.xpath('//oai:record', namespaces = namespaces):
                sources = [
                    dc_source.text.strip()or None if dc_source.text is not None else None
                    for dc_source in oai_record.xpath('.//dc:source', namespaces = namespaces)
                    ]
                if len(sources) >= 2:
                    location = sources[1]
                    if location is not None:
                        locations.setdefault(location, set()).add(organization_name)
                subjects = [
                    dc_subject.text.strip()or None if dc_subject.text is not None else None
                    for dc_subject in oai_record.xpath('.//dc:subject', namespaces = namespaces)
                    ]
                if subjects:
                    theme = subjects[0]
                    if theme is not None:
                        sub_theme = subjects[1] or None if len(subjects) >= 2 else None
                        themes.setdefault(theme, {}).setdefault(sub_theme, set()).add(organization_name)
                    if len(subjects) >= 3:
                        for keyword in subjects[2:]:
                            keywords.setdefault(keyword, set()).add(organization_name)
            xml_resumption_token_list = oai_pmh_doc.xpath('//oai:resumptionToken', namespaces = dict(
                oai = 'http://www.openarchives.org/OAI/2.0/',
                ))
            if not xml_resumption_token_list:
                break
            resumption_token = xml_resumption_token_list[0].text
            if not resumption_token:
                break
            log.info('Continuing with resumption token {0}'.format(resumption_token))

    keywords_file = codecs.open(os.path.join(dir, 'mots-cles.txt'), 'w', encoding = 'utf-8')
    for keyword in sorted(keywords, key = lambda s: ((s or '').lower(), s)):
        error = u'Trop de caractères: {0} au lieu de 50'.format(len(keyword)) if len(keyword) > 50 else None
        organizations_name = sorted(keywords[keyword], key = lambda s: ((s or '').lower(), s))
        keywords_file.write(u'|| {0} || {1} ||{2} ||\n'.format(
            keyword.replace(u'\n', u'\\n'), u', '.join(organizations_name), error or ''))
    keywords_file.close()

    locations_file = codecs.open(os.path.join(dir, 'lieux.txt'), 'w', encoding = 'utf-8')
    for location in sorted(locations, key = lambda s: ((s or '').lower(), s)):
        error = u'Trop de caractères: {0} au lieu de 255'.format(len(location)) if len(location) > 255 else None
        organizations_name = sorted(locations[location], key = lambda s: ((s or '').lower(), s))
        locations_file.write(u'|| {0} || {1} ||{2} ||\n'.format(
            location.replace(u'\n', u'\\n'), u', '.join(organizations_name), error or ''))
    locations_file.close()

    themes_and_sub_themes_file = codecs.open(os.path.join(dir, 'themes-sous-themes.txt'), 'w', encoding = 'utf-8')
    themes_file = codecs.open(os.path.join(dir, 'themes.txt'), 'w', encoding = 'utf-8')
    for theme in sorted(themes, key = lambda s: ((s or '').lower(), s)):
        for sub_theme in sorted(themes[theme], key = lambda s: ((s or '').lower(), s)):
            themes_and_sub_themes_file.write(u'{0}|{1}\n'.format(
                theme.replace(u'\n', u' - '), (sub_theme or '').replace(u'\n', u' - ')))
            organizations_name = sorted(themes[theme][sub_theme], key = lambda s: ((s or '').lower(), s))
            themes_file.write(u'|| {0} || {1} ||{2} ||\n'.format(
                theme.replace(u'\n', u'\\n'), (sub_theme or '').replace(u'\n', u'\\n'), u', '.join(organizations_name)))
    themes_and_sub_themes_file.close()
    themes_file.close()

    return 0


if __name__ == "__main__":
    sys.exit(main())
